const Crawler = require("crawler");
const {crawlerConfig} = require('../config')
const logger = require('../utils/logger')
const totalInfoHandle = require('../utils/totalInfoHandle')
const c = new Crawler({
  ...{crawlerConfig},
  callback: function (error, res, done) {
    if(error){
      totalInfoHandle.addError({
        type: 3,
        url: res.options.uri
      })
    }
    done();
  }
});

c.on('request',function(options){

});

c.on('drain',function(){
  logger.info('------------ crawl queue empty ------------')
  totalInfoHandle.show()
  logger.info('------------ crawl queue empty ------------')
});

c.on('schedule',function(options){
  totalInfoHandle.add({
    type: 4,
    url: options.uri
  })
});

module.exports = c
